#!/usr/bin/php
<?
//
// Populate a Redis table with the keys of all existing storage files
//
// This needs to be run before storage_purge.
//
set_include_path("../include");
require("header.inc.php");

define('SET_NAME_EXISTING', 'storage:keys:existing');

$startShardID = empty($argv[1]) ? 0 : $argv[1];

$info = Zotero_DBConnectAuth('master');
$masterDB = new mysqli($info['host'], $info['user'], $info['pass'], $info['db'], $info['port']);
$masterDB->set_charset('utf8');

$redis = Z_Redis::get();

// Empty storageFilesExisting table if starting at the beginning
if (!$startShardID) {
	echo "Emptying storageFilesExisting\n";
	sleep(2);
	Zotero_Admin_DB::query("TRUNCATE TABLE storageFilesExisting");
	
	echo "Emptying " . SET_NAME_EXISTING . "\n";
	$redis->del(SET_NAME_EXISTING);
}

$maxInsertGroups = 150;

// Get a list of all used files on all shards and add to a table on master
$shardIDs = Zotero_DB::columnQuery("SELECT shardID FROM shards S WHERE shardID>=? ORDER BY shardID", array($startShardID));
foreach ($shardIDs as $shardID) {
	echo "Shard: $shardID\n";
	
	$sql = "SELECT DISTINCT storageFileID FROM storageFileItems";
	$ids = Zotero_DB::columnQuery($sql, false, $shardID);
	
	$inserted = 0;
	$origInsertSQL = "INSERT IGNORE INTO storageFilesExisting VALUES ";
	
	while ($insertIDs = array_splice($ids, 0, $maxInsertGroups)) {
		$num = sizeOf($insertIDs);
		$insertSQL = $origInsertSQL . implode(',', array_fill(0, $num, '(?)'));
		$stmt = Zotero_DB::getStatement($insertSQL, true);
		Zotero_DB::queryFromStatement($stmt, $insertIDs);
		$inserted += $num;
	}
	
	echo "Inserted " . $inserted . " storageFileIDs\n";
}

//
// Join against storageFiles to get the S3 key names
//
$info = Zotero_DBConnectAuth('master');
$masterDB = new mysqli($info['host'], $info['user'], $info['pass'], $info['db'], $info['port']);
$masterDB->set_charset('utf8');

$sql = "SELECT CONCAT(hash, '/', IF(zip=1, 'c/', ''), filename) AS `key` "
	. "FROM storageFilesExisting JOIN storageFiles USING (storageFileID) ORDER BY storageFileID LIMIT ";
$offset = 0;
$prefetch = 50000;
$numRows = 100000;
while (true) {
	echo "Getting $numRows rows starting at $offset\n";
	
	$stmt = $masterDB->prepare($sql . "$offset, $numRows") or die($masterDB->error);
	$stmt->attr_set(MYSQLI_STMT_ATTR_CURSOR_TYPE, MYSQLI_CURSOR_TYPE_READ_ONLY);
	$stmt->attr_set(MYSQLI_STMT_ATTR_PREFETCH_ROWS, $prefetch);
	$stmt->execute();
	$stmt->bind_result($key);
	
	// Add keys to redis set
	$i = 0;
	$arr = [SET_NAME_EXISTING];
	while ($stmt->fetch()) {
		$arr[] = $key;
		$i++;
		if ($i % 50 == 0) {
			call_user_func_array([$redis, "sadd"], $arr);
			$arr = [SET_NAME_EXISTING];
		}
	}
	if (sizeOf($arr) > 1) {
		call_user_func_array([$redis, "sadd"], $arr);
	}
	$stmt->close();
	
	if ($i == 0) {
		break;
	}
	
	$offset += $numRows;
}

// Join against storageFiles to get the S3 key names
$sql = "SELECT CONCAT(hash, '/', IF(zip=1, 'c/', ''), filename) AS `key` "
	. "FROM storageFilesExisting JOIN storageFiles USING (storageFileID) ORDER BY storageFileID";
$stmt = $masterDB->prepare($sql) or die($masterDB->error);
$stmt->attr_set(MYSQLI_STMT_ATTR_CURSOR_TYPE, MYSQLI_CURSOR_TYPE_READ_ONLY);
$stmt->attr_set(MYSQLI_STMT_ATTR_PREFETCH_ROWS, 50000);
$stmt->execute();
$stmt->bind_result($key);

// Add keys to redis set
$i = 0;
$arr = [SET_NAME_EXISTING];
while ($stmt->fetch()) {
	$arr[] = $key;
	$i++;
	if ($i % 50 == 0) {
		call_user_func_array([$redis, "sadd"], $arr);
		$arr = [SET_NAME_EXISTING];
	}
}
if (sizeOf($arr) > 1) {
	call_user_func_array([$redis, "sadd"], $arr);
}
$stmt->close();

$masterDB->close();
?>
